

<!DOCTYPE html>
<html class="writer-html5" lang="zh" >
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>扩展 &mdash; Scrapy 2.3.0 文档</title>
  

  
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster.custom.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster.bundle.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-shadow.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-punk.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-noir.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-light.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-borderless.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/micromodal.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/sphinx_rtd_theme.css" type="text/css" />

  
  
  
  

  
  <!--[if lt IE 9]>
    <script src="../_static/js/html5shiv.min.js"></script>
  <![endif]-->
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
        <script src="../_static/jquery.js"></script>
        <script src="../_static/underscore.js"></script>
        <script src="../_static/doctools.js"></script>
        <script src="../_static/language_data.js"></script>
        <script src="../_static/js/hoverxref.js"></script>
        <script src="../_static/js/tooltipster.bundle.min.js"></script>
        <script src="../_static/js/micromodal.min.js"></script>
    
    <script type="text/javascript" src="../_static/js/theme.js"></script>

    
    <link rel="index" title="索引" href="../genindex.html" />
    <link rel="search" title="搜索" href="../search.html" />
    <link rel="next" title="核心API" href="api.html" />
    <link rel="prev" title="蜘蛛中间件" href="spider-middleware.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../index.html" class="icon icon-home" alt="Documentation Home"> Scrapy
          

          
          </a>

          
            
            
              <div class="version">
                2.3
              </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">第一步</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../intro/overview.html">Scrapy一目了然</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intro/install.html">安装指南</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intro/tutorial.html">Scrapy 教程</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intro/examples.html">实例</a></li>
</ul>
<p class="caption"><span class="caption-text">基本概念</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="commands.html">命令行工具</a></li>
<li class="toctree-l1"><a class="reference internal" href="spiders.html">蜘蛛</a></li>
<li class="toctree-l1"><a class="reference internal" href="selectors.html">选择器</a></li>
<li class="toctree-l1"><a class="reference internal" href="items.html">项目</a></li>
<li class="toctree-l1"><a class="reference internal" href="loaders.html">项目加载器</a></li>
<li class="toctree-l1"><a class="reference internal" href="shell.html">Scrapy shell</a></li>
<li class="toctree-l1"><a class="reference internal" href="item-pipeline.html">项目管道</a></li>
<li class="toctree-l1"><a class="reference internal" href="feed-exports.html">Feed 导出</a></li>
<li class="toctree-l1"><a class="reference internal" href="request-response.html">请求和响应</a></li>
<li class="toctree-l1"><a class="reference internal" href="link-extractors.html">链接提取器</a></li>
<li class="toctree-l1"><a class="reference internal" href="settings.html">设置</a></li>
<li class="toctree-l1"><a class="reference internal" href="exceptions.html">例外情况</a></li>
</ul>
<p class="caption"><span class="caption-text">内置服务</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="logging.html">登录</a></li>
<li class="toctree-l1"><a class="reference internal" href="stats.html">统计数据集合</a></li>
<li class="toctree-l1"><a class="reference internal" href="email.html">发送电子邮件</a></li>
<li class="toctree-l1"><a class="reference internal" href="telnetconsole.html">远程登录控制台</a></li>
<li class="toctree-l1"><a class="reference internal" href="webservice.html">Web服务</a></li>
</ul>
<p class="caption"><span class="caption-text">解决具体问题</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../faq.html">常见问题</a></li>
<li class="toctree-l1"><a class="reference internal" href="debug.html">调试spiders</a></li>
<li class="toctree-l1"><a class="reference internal" href="contracts.html">蜘蛛合约</a></li>
<li class="toctree-l1"><a class="reference internal" href="practices.html">常用做法</a></li>
<li class="toctree-l1"><a class="reference internal" href="broad-crawls.html">宽爬行</a></li>
<li class="toctree-l1"><a class="reference internal" href="developer-tools.html">使用浏览器的开发人员工具进行抓取</a></li>
<li class="toctree-l1"><a class="reference internal" href="dynamic-content.html">选择动态加载的内容</a></li>
<li class="toctree-l1"><a class="reference internal" href="leaks.html">调试内存泄漏</a></li>
<li class="toctree-l1"><a class="reference internal" href="media-pipeline.html">下载和处理文件和图像</a></li>
<li class="toctree-l1"><a class="reference internal" href="deploy.html">部署蜘蛛</a></li>
<li class="toctree-l1"><a class="reference internal" href="autothrottle.html">AutoThrottle 扩展</a></li>
<li class="toctree-l1"><a class="reference internal" href="benchmarking.html">标杆管理</a></li>
<li class="toctree-l1"><a class="reference internal" href="jobs.html">作业：暂停和恢复爬行</a></li>
<li class="toctree-l1"><a class="reference internal" href="coroutines.html">协同程序</a></li>
<li class="toctree-l1"><a class="reference internal" href="asyncio.html">asyncio</a></li>
</ul>
<p class="caption"><span class="caption-text">扩展Scrapy</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="architecture.html">体系结构概述</a></li>
<li class="toctree-l1"><a class="reference internal" href="downloader-middleware.html">下载器中间件</a></li>
<li class="toctree-l1"><a class="reference internal" href="spider-middleware.html">蜘蛛中间件</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">扩展</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#extension-settings">扩展设置</a></li>
<li class="toctree-l2"><a class="reference internal" href="#loading-activating-extensions">加载和激活扩展</a></li>
<li class="toctree-l2"><a class="reference internal" href="#available-enabled-and-disabled-extensions">可用、启用和禁用扩展</a></li>
<li class="toctree-l2"><a class="reference internal" href="#disabling-an-extension">禁用扩展</a></li>
<li class="toctree-l2"><a class="reference internal" href="#writing-your-own-extension">编写自己的扩展名</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#sample-extension">样本扩展</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#built-in-extensions-reference">内置扩展引用</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#general-purpose-extensions">通用扩展</a><ul>
<li class="toctree-l4"><a class="reference internal" href="#module-scrapy.extensions.logstats">日志统计扩展</a></li>
<li class="toctree-l4"><a class="reference internal" href="#module-scrapy.extensions.corestats">核心统计扩展</a></li>
<li class="toctree-l4"><a class="reference internal" href="#module-scrapy.extensions.telnet">Telnet控制台扩展</a></li>
<li class="toctree-l4"><a class="reference internal" href="#module-scrapy.extensions.memusage">内存使用扩展</a></li>
<li class="toctree-l4"><a class="reference internal" href="#module-scrapy.extensions.memdebug">内存调试器扩展</a></li>
<li class="toctree-l4"><a class="reference internal" href="#module-scrapy.extensions.closespider">关闭星形延长件</a></li>
<li class="toctree-l4"><a class="reference internal" href="#module-scrapy.extensions.statsmailer">StatsMailer扩展</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="#debugging-extensions">调试扩展</a><ul>
<li class="toctree-l4"><a class="reference internal" href="#stack-trace-dump-extension">堆栈跟踪转储扩展</a></li>
<li class="toctree-l4"><a class="reference internal" href="#debugger-extension">调试器扩展</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="api.html">核心API</a></li>
<li class="toctree-l1"><a class="reference internal" href="signals.html">信号</a></li>
<li class="toctree-l1"><a class="reference internal" href="exporters.html">条目导出器</a></li>
</ul>
<p class="caption"><span class="caption-text">其余所有</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../news.html">发行说明</a></li>
<li class="toctree-l1"><a class="reference internal" href="../contributing.html">为 Scrapy 贡献</a></li>
<li class="toctree-l1"><a class="reference internal" href="../versioning.html">版本控制和API稳定性</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../index.html">Scrapy</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
        
      <li>扩展</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="extensions">
<span id="topics-extensions"></span><h1>扩展<a class="headerlink" href="#extensions" title="永久链接至标题">¶</a></h1>
<p>扩展框架提供了一种将您自己的自定义功能插入到Scrapy中的机制。</p>
<p>扩展只是在初始化扩展时，在scrapy启动时实例化的常规类。</p><script async src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js"></script>
<ins class="adsbygoogle"
     style="display:block; text-align:center;"
     data-ad-layout="in-article"
     data-ad-format="fluid"
     data-ad-client="ca-pub-1466963416408457"
     data-ad-slot="8850786025"></ins>
<script>
     (adsbygoogle = window.adsbygoogle || []).push({});
</script>
<div class="section" id="extension-settings">
<h2>扩展设置<a class="headerlink" href="#extension-settings" title="永久链接至标题">¶</a></h2>
<p>扩展使用 <a class="reference internal" href="settings.html#topics-settings"><span class="std std-ref">Scrapy settings</span></a> 管理他们的设置，就像任何其他零碎的代码一样。</p>
<p>扩展通常会在其设置前面加上自己的名称，以避免与现有（和将来）扩展冲突。例如，要处理的假设扩展 <a class="reference external" href="https://en.wikipedia.org/wiki/Sitemaps">Google Sitemaps</a> 将使用如下设置 <code class="docutils literal notranslate"><span class="pre">GOOGLESITEMAP_ENABLED</span></code> ， <code class="docutils literal notranslate"><span class="pre">GOOGLESITEMAP_DEPTH</span></code> 等等。</p>
</div>
<div class="section" id="loading-activating-extensions">
<h2>加载和激活扩展<a class="headerlink" href="#loading-activating-extensions" title="永久链接至标题">¶</a></h2>
<p>扩展在启动时通过实例化扩展类的单个实例来加载和激活。因此，所有的扩展初始化代码都必须在类中执行 <code class="docutils literal notranslate"><span class="pre">__init__</span></code> 方法。</p>
<p>若要使扩展名可用，请将其添加到 <a class="reference internal" href="settings.html#std-setting-EXTENSIONS"><code class="xref std std-setting docutils literal notranslate"><span class="pre">EXTENSIONS</span></code></a> 设置你的剪贴设置。在 <a class="reference internal" href="settings.html#std-setting-EXTENSIONS"><code class="xref std std-setting docutils literal notranslate"><span class="pre">EXTENSIONS</span></code></a> ，每个扩展都由一个字符串表示：扩展类名的完整python路径。例如：：</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">EXTENSIONS</span> <span class="o">=</span> <span class="p">{</span>
    <span class="s1">&#39;scrapy.extensions.corestats.CoreStats&#39;</span><span class="p">:</span> <span class="mi">500</span><span class="p">,</span>
    <span class="s1">&#39;scrapy.extensions.telnet.TelnetConsole&#39;</span><span class="p">:</span> <span class="mi">500</span><span class="p">,</span>
<span class="p">}</span>
</pre></div>
</div>
<p>如你所见， <a class="reference internal" href="settings.html#std-setting-EXTENSIONS"><code class="xref std std-setting docutils literal notranslate"><span class="pre">EXTENSIONS</span></code></a> 设置是一个dict，其中键是扩展路径，它们的值是定义扩展的顺序。 <em>加载</em> 秩序。这个 <a class="reference internal" href="settings.html#std-setting-EXTENSIONS"><code class="xref std std-setting docutils literal notranslate"><span class="pre">EXTENSIONS</span></code></a> 设置与合并 <a class="reference internal" href="settings.html#std-setting-EXTENSIONS_BASE"><code class="xref std std-setting docutils literal notranslate"><span class="pre">EXTENSIONS_BASE</span></code></a> 在scrappy中定义的设置（不打算被重写），然后按顺序排序，以获得已启用扩展的最终排序列表。</p>
<p>由于扩展通常不相互依赖，因此在大多数情况下，它们的加载顺序是不相关的。这就是为什么 <a class="reference internal" href="settings.html#std-setting-EXTENSIONS_BASE"><code class="xref std std-setting docutils literal notranslate"><span class="pre">EXTENSIONS_BASE</span></code></a> 设置以相同的顺序定义所有扩展名 (<code class="docutils literal notranslate"><span class="pre">0</span></code> ）但是，如果需要添加依赖于已加载的其他扩展的扩展，则可以利用此功能。</p>
</div>
<div class="section" id="available-enabled-and-disabled-extensions">
<h2>可用、启用和禁用扩展<a class="headerlink" href="#available-enabled-and-disabled-extensions" title="永久链接至标题">¶</a></h2>
<p>并非所有可用的扩展都将启用。其中一些通常取决于特定的设置。例如，HTTP缓存扩展在默认情况下是可用的，但在 <a class="reference internal" href="downloader-middleware.html#std-setting-HTTPCACHE_ENABLED"><code class="xref std std-setting docutils literal notranslate"><span class="pre">HTTPCACHE_ENABLED</span></code></a> 设置设置。</p>
</div>
<div class="section" id="disabling-an-extension">
<h2>禁用扩展<a class="headerlink" href="#disabling-an-extension" title="永久链接至标题">¶</a></h2>
<p>为了禁用默认启用的扩展（即 <a class="reference internal" href="settings.html#std-setting-EXTENSIONS_BASE"><code class="xref std std-setting docutils literal notranslate"><span class="pre">EXTENSIONS_BASE</span></code></a> 设置）必须将其顺序设置为 <code class="docutils literal notranslate"><span class="pre">None</span></code> . 例如：：</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">EXTENSIONS</span> <span class="o">=</span> <span class="p">{</span>
    <span class="s1">&#39;scrapy.extensions.corestats.CoreStats&#39;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="p">}</span>
</pre></div>
</div>
</div>
<div class="section" id="writing-your-own-extension">
<h2>编写自己的扩展名<a class="headerlink" href="#writing-your-own-extension" title="永久链接至标题">¶</a></h2>
<p>每个扩展都是一个Python类。碎屑扩展（也包括中间产品和管道）的主要入口点是 <code class="docutils literal notranslate"><span class="pre">from_crawler</span></code> 类方法，它接收 <code class="docutils literal notranslate"><span class="pre">Crawler</span></code> 实例。通过爬虫对象，您可以访问设置、信号、统计信息，还可以控制爬行行为。</p>
<p>通常，扩展连接到 <a class="reference internal" href="signals.html#topics-signals"><span class="std std-ref">signals</span></a> 并执行由它们触发的任务。</p>
<p>最后，如果 <code class="docutils literal notranslate"><span class="pre">from_crawler</span></code> 方法引发 <a class="reference internal" href="exceptions.html#scrapy.exceptions.NotConfigured" title="scrapy.exceptions.NotConfigured"><code class="xref py py-exc docutils literal notranslate"><span class="pre">NotConfigured</span></code></a> 异常，扩展将被禁用。否则，将启用扩展。</p>
<div class="section" id="sample-extension">
<h3>样本扩展<a class="headerlink" href="#sample-extension" title="永久链接至标题">¶</a></h3>
<p>在这里，我们将实现一个简单的扩展来说明前一节中描述的概念。此扩展将每次记录一条消息：</p>
<ul class="simple">
<li><p>蜘蛛被打开了</p></li>
<li><p>蜘蛛被关闭了</p></li>
<li><p>特定数量的物品被爬取</p></li>
</ul>
<p>扩展将通过 <code class="docutils literal notranslate"><span class="pre">MYEXT_ENABLED</span></code> 设置和项目数将通过 <code class="docutils literal notranslate"><span class="pre">MYEXT_ITEMCOUNT</span></code> 设置。</p>
<p>这是这种扩展的代码：</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">from</span> <span class="nn">scrapy</span> <span class="kn">import</span> <span class="n">signals</span>
<span class="kn">from</span> <span class="nn">scrapy.exceptions</span> <span class="kn">import</span> <span class="n">NotConfigured</span>

<span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>

<span class="k">class</span> <span class="nc">SpiderOpenCloseLogging</span><span class="p">:</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item_count</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">item_count</span> <span class="o">=</span> <span class="n">item_count</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">items_scraped</span> <span class="o">=</span> <span class="mi">0</span>

    <span class="nd">@classmethod</span>
    <span class="k">def</span> <span class="nf">from_crawler</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">crawler</span><span class="p">):</span>
        <span class="c1"># first check if the extension should be enabled and raise</span>
        <span class="c1"># NotConfigured otherwise</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="n">crawler</span><span class="o">.</span><span class="n">settings</span><span class="o">.</span><span class="n">getbool</span><span class="p">(</span><span class="s1">&#39;MYEXT_ENABLED&#39;</span><span class="p">):</span>
            <span class="k">raise</span> <span class="n">NotConfigured</span>

        <span class="c1"># get the number of items from settings</span>
        <span class="n">item_count</span> <span class="o">=</span> <span class="n">crawler</span><span class="o">.</span><span class="n">settings</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">&#39;MYEXT_ITEMCOUNT&#39;</span><span class="p">,</span> <span class="mi">1000</span><span class="p">)</span>

        <span class="c1"># instantiate the extension object</span>
        <span class="n">ext</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">(</span><span class="n">item_count</span><span class="p">)</span>

        <span class="c1"># connect the extension object to signals</span>
        <span class="n">crawler</span><span class="o">.</span><span class="n">signals</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">ext</span><span class="o">.</span><span class="n">spider_opened</span><span class="p">,</span> <span class="n">signal</span><span class="o">=</span><span class="n">signals</span><span class="o">.</span><span class="n">spider_opened</span><span class="p">)</span>
        <span class="n">crawler</span><span class="o">.</span><span class="n">signals</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">ext</span><span class="o">.</span><span class="n">spider_closed</span><span class="p">,</span> <span class="n">signal</span><span class="o">=</span><span class="n">signals</span><span class="o">.</span><span class="n">spider_closed</span><span class="p">)</span>
        <span class="n">crawler</span><span class="o">.</span><span class="n">signals</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">ext</span><span class="o">.</span><span class="n">item_scraped</span><span class="p">,</span> <span class="n">signal</span><span class="o">=</span><span class="n">signals</span><span class="o">.</span><span class="n">item_scraped</span><span class="p">)</span>

        <span class="c1"># return the extension object</span>
        <span class="k">return</span> <span class="n">ext</span>

    <span class="k">def</span> <span class="nf">spider_opened</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">spider</span><span class="p">):</span>
        <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;opened spider </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">spider</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">spider_closed</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">spider</span><span class="p">):</span>
        <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;closed spider </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">spider</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">item_scraped</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">,</span> <span class="n">spider</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">items_scraped</span> <span class="o">+=</span> <span class="mi">1</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">items_scraped</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">item_count</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
            <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;scraped </span><span class="si">%d</span><span class="s2"> items&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">items_scraped</span><span class="p">)</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="built-in-extensions-reference">
<span id="topics-extensions-ref"></span><h2>内置扩展引用<a class="headerlink" href="#built-in-extensions-reference" title="永久链接至标题">¶</a></h2>
<div class="section" id="general-purpose-extensions">
<h3>通用扩展<a class="headerlink" href="#general-purpose-extensions" title="永久链接至标题">¶</a></h3>
<div class="section" id="module-scrapy.extensions.logstats">
<span id="log-stats-extension"></span><h4>日志统计扩展<a class="headerlink" href="#module-scrapy.extensions.logstats" title="永久链接至标题">¶</a></h4>
<dl class="py class">
<dt id="scrapy.extensions.logstats.LogStats">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.extensions.logstats.</code><code class="sig-name descname">LogStats</code><a class="reference internal" href="../_modules/scrapy/extensions/logstats.html#LogStats"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.extensions.logstats.LogStats" title="永久链接至目标">¶</a></dt>
<dd></dd></dl>

<p>记录基本的统计信息，如已爬网的页面和已擦除的项目。</p>
</div>
<div class="section" id="module-scrapy.extensions.corestats">
<span id="core-stats-extension"></span><h4>核心统计扩展<a class="headerlink" href="#module-scrapy.extensions.corestats" title="永久链接至标题">¶</a></h4>
<dl class="py class">
<dt id="scrapy.extensions.corestats.CoreStats">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.extensions.corestats.</code><code class="sig-name descname">CoreStats</code><a class="reference internal" href="../_modules/scrapy/extensions/corestats.html#CoreStats"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.extensions.corestats.CoreStats" title="永久链接至目标">¶</a></dt>
<dd></dd></dl>

<p>启用核心统计信息集合，前提是已启用统计信息集合（请参见 <a class="reference internal" href="stats.html#topics-stats"><span class="std std-ref">统计数据集合</span></a> ）</p>
</div>
<div class="section" id="module-scrapy.extensions.telnet">
<span id="telnet-console-extension"></span><span id="topics-extensions-ref-telnetconsole"></span><h4>Telnet控制台扩展<a class="headerlink" href="#module-scrapy.extensions.telnet" title="永久链接至标题">¶</a></h4>
<dl class="py class">
<dt id="scrapy.extensions.telnet.TelnetConsole">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.extensions.telnet.</code><code class="sig-name descname">TelnetConsole</code><a class="reference internal" href="../_modules/scrapy/extensions/telnet.html#TelnetConsole"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.extensions.telnet.TelnetConsole" title="永久链接至目标">¶</a></dt>
<dd></dd></dl>

<p>提供一个telnet控制台，用于进入当前运行的scrapy进程中的python解释器，这对于调试非常有用。</p>
<p>telnet控制台必须由 <a class="reference internal" href="settings.html#std-setting-TELNETCONSOLE_ENABLED"><code class="xref std std-setting docutils literal notranslate"><span class="pre">TELNETCONSOLE_ENABLED</span></code></a> 设置，服务器将侦听中指定的端口 <a class="reference internal" href="telnetconsole.html#std-setting-TELNETCONSOLE_PORT"><code class="xref std std-setting docutils literal notranslate"><span class="pre">TELNETCONSOLE_PORT</span></code></a> .</p>
</div>
<div class="section" id="module-scrapy.extensions.memusage">
<span id="memory-usage-extension"></span><span id="topics-extensions-ref-memusage"></span><h4>内存使用扩展<a class="headerlink" href="#module-scrapy.extensions.memusage" title="永久链接至标题">¶</a></h4>
<dl class="py class">
<dt id="scrapy.extensions.memusage.MemoryUsage">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.extensions.memusage.</code><code class="sig-name descname">MemoryUsage</code><a class="headerlink" href="#scrapy.extensions.memusage.MemoryUsage" title="永久链接至目标">¶</a></dt>
<dd></dd></dl>

<div class="admonition note">
<p class="admonition-title">注解</p>
<p>此扩展在Windows中不起作用。</p>
</div>
<p>监视运行spider的scrapy进程使用的内存，并：</p>
<ol class="arabic simple">
<li><p>超过某个值时发送通知电子邮件</p></li>
<li><p>当蜘蛛超过某个值时关闭蜘蛛</p></li>
</ol>
<p>当达到某个警告值时，可以触发通知电子邮件。 (<a class="reference internal" href="settings.html#std-setting-MEMUSAGE_WARNING_MB"><code class="xref std std-setting docutils literal notranslate"><span class="pre">MEMUSAGE_WARNING_MB</span></code></a> ）当达到最大值时 (<a class="reference internal" href="settings.html#std-setting-MEMUSAGE_LIMIT_MB"><code class="xref std std-setting docutils literal notranslate"><span class="pre">MEMUSAGE_LIMIT_MB</span></code></a> ）这也会导致蜘蛛被关闭，碎屑过程被终止。</p>
<p>此扩展由 <a class="reference internal" href="settings.html#std-setting-MEMUSAGE_ENABLED"><code class="xref std std-setting docutils literal notranslate"><span class="pre">MEMUSAGE_ENABLED</span></code></a> 设置，可以使用以下设置进行配置：</p>
<ul class="simple">
<li><p><a class="reference internal" href="settings.html#std-setting-MEMUSAGE_LIMIT_MB"><code class="xref std std-setting docutils literal notranslate"><span class="pre">MEMUSAGE_LIMIT_MB</span></code></a></p></li>
<li><p><a class="reference internal" href="settings.html#std-setting-MEMUSAGE_WARNING_MB"><code class="xref std std-setting docutils literal notranslate"><span class="pre">MEMUSAGE_WARNING_MB</span></code></a></p></li>
<li><p><a class="reference internal" href="settings.html#std-setting-MEMUSAGE_NOTIFY_MAIL"><code class="xref std std-setting docutils literal notranslate"><span class="pre">MEMUSAGE_NOTIFY_MAIL</span></code></a></p></li>
<li><p><a class="reference internal" href="settings.html#std-setting-MEMUSAGE_CHECK_INTERVAL_SECONDS"><code class="xref std std-setting docutils literal notranslate"><span class="pre">MEMUSAGE_CHECK_INTERVAL_SECONDS</span></code></a></p></li>
</ul>
</div>
<div class="section" id="module-scrapy.extensions.memdebug">
<span id="memory-debugger-extension"></span><h4>内存调试器扩展<a class="headerlink" href="#module-scrapy.extensions.memdebug" title="永久链接至标题">¶</a></h4>
<dl class="py class">
<dt id="scrapy.extensions.memdebug.MemoryDebugger">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.extensions.memdebug.</code><code class="sig-name descname">MemoryDebugger</code><a class="reference internal" href="../_modules/scrapy/extensions/memdebug.html#MemoryDebugger"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.extensions.memdebug.MemoryDebugger" title="永久链接至目标">¶</a></dt>
<dd></dd></dl>

<p>调试内存使用情况的扩展。它收集以下信息：</p>
<ul class="simple">
<li><p>python垃圾收集器未收集的对象</p></li>
<li><p>不应该保留的对象。有关详细信息，请参阅 <a class="reference internal" href="leaks.html#topics-leaks-trackrefs"><span class="std std-ref">使用调试内存泄漏 trackref</span></a></p></li>
</ul>
<p>要启用此扩展，请打开 <a class="reference internal" href="settings.html#std-setting-MEMDEBUG_ENABLED"><code class="xref std std-setting docutils literal notranslate"><span class="pre">MEMDEBUG_ENABLED</span></code></a> 设置。信息将存储在统计信息中。</p>
</div>
<div class="section" id="module-scrapy.extensions.closespider">
<span id="close-spider-extension"></span><h4>关闭星形延长件<a class="headerlink" href="#module-scrapy.extensions.closespider" title="永久链接至标题">¶</a></h4>
<dl class="py class">
<dt id="scrapy.extensions.closespider.CloseSpider">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.extensions.closespider.</code><code class="sig-name descname">CloseSpider</code><a class="reference internal" href="../_modules/scrapy/extensions/closespider.html#CloseSpider"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.extensions.closespider.CloseSpider" title="永久链接至目标">¶</a></dt>
<dd></dd></dl>

<p>当满足某些条件时，使用每个条件的特定关闭原因自动关闭蜘蛛。</p>
<p>关闭蜘蛛的条件可以通过以下设置进行配置：</p>
<ul class="simple">
<li><p><a class="reference internal" href="#std-setting-CLOSESPIDER_TIMEOUT"><code class="xref std std-setting docutils literal notranslate"><span class="pre">CLOSESPIDER_TIMEOUT</span></code></a></p></li>
<li><p><a class="reference internal" href="#std-setting-CLOSESPIDER_ITEMCOUNT"><code class="xref std std-setting docutils literal notranslate"><span class="pre">CLOSESPIDER_ITEMCOUNT</span></code></a></p></li>
<li><p><a class="reference internal" href="#std-setting-CLOSESPIDER_PAGECOUNT"><code class="xref std std-setting docutils literal notranslate"><span class="pre">CLOSESPIDER_PAGECOUNT</span></code></a></p></li>
<li><p><a class="reference internal" href="#std-setting-CLOSESPIDER_ERRORCOUNT"><code class="xref std std-setting docutils literal notranslate"><span class="pre">CLOSESPIDER_ERRORCOUNT</span></code></a></p></li>
</ul>
<div class="section" id="closespider-timeout">
<span id="std-setting-CLOSESPIDER_TIMEOUT"></span><span id="std:setting-CLOSESPIDER_TIMEOUT"></span><h5>CLOSESPIDER_TIMEOUT<a class="headerlink" href="#closespider-timeout" title="永久链接至标题">¶</a></h5>
<p>违约： <code class="docutils literal notranslate"><span class="pre">0</span></code></p>
<p>指定秒数的整数。如果蜘蛛保持打开超过该秒数，它将自动关闭，原因如下 <code class="docutils literal notranslate"><span class="pre">closespider_timeout</span></code> . 如果为零（或未设置），蜘蛛将不会在超时时关闭。</p>
</div>
<div class="section" id="closespider-itemcount">
<span id="std-setting-CLOSESPIDER_ITEMCOUNT"></span><span id="std:setting-CLOSESPIDER_ITEMCOUNT"></span><h5>CLOSESPIDER_ITEMCOUNT<a class="headerlink" href="#closespider-itemcount" title="永久链接至标题">¶</a></h5>
<p>违约： <code class="docutils literal notranslate"><span class="pre">0</span></code></p>
<p>指定若干项的整数。如果spider的爬取超过该数量，并且这些项目通过项目管道，那么spider将关闭，原因是 <code class="docutils literal notranslate"><span class="pre">closespider_itemcount</span></code> . 当前在下载器队列中的请求（最多 <a class="reference internal" href="settings.html#std-setting-CONCURRENT_REQUESTS"><code class="xref std std-setting docutils literal notranslate"><span class="pre">CONCURRENT_REQUESTS</span></code></a> 请求）仍在处理中。如果为零（或未设置），蜘蛛将不会被通过的项目数关闭。</p>
</div>
<div class="section" id="closespider-pagecount">
<span id="std-setting-CLOSESPIDER_PAGECOUNT"></span><span id="std:setting-CLOSESPIDER_PAGECOUNT"></span><h5>CLOSESPIDER_PAGECOUNT<a class="headerlink" href="#closespider-pagecount" title="永久链接至标题">¶</a></h5>
<p>违约： <code class="docutils literal notranslate"><span class="pre">0</span></code></p>
<p>指定要爬网的最大响应数的整数。如果蜘蛛爬得比这个还多，蜘蛛会被关闭的，原因是 <code class="docutils literal notranslate"><span class="pre">closespider_pagecount</span></code> . 如果为零（或未设置），蜘蛛将不会被爬行响应的数量关闭。</p>
</div>
<div class="section" id="closespider-errorcount">
<span id="std-setting-CLOSESPIDER_ERRORCOUNT"></span><span id="std:setting-CLOSESPIDER_ERRORCOUNT"></span><h5>CLOSESPIDER_ERRORCOUNT<a class="headerlink" href="#closespider-errorcount" title="永久链接至标题">¶</a></h5>
<p>违约： <code class="docutils literal notranslate"><span class="pre">0</span></code></p>
<p>一个整数，指定关闭蜘蛛之前要接收的最大错误数。如果spider生成的错误数量超过该数量，则会关闭它并说明原因。 <code class="docutils literal notranslate"><span class="pre">closespider_errorcount</span></code> . 如果为零（或未设置），蜘蛛将不会被错误数关闭。</p>
</div>
</div>
<div class="section" id="module-scrapy.extensions.statsmailer">
<span id="statsmailer-extension"></span><h4>StatsMailer扩展<a class="headerlink" href="#module-scrapy.extensions.statsmailer" title="永久链接至标题">¶</a></h4>
<dl class="py class">
<dt id="scrapy.extensions.statsmailer.StatsMailer">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.extensions.statsmailer.</code><code class="sig-name descname">StatsMailer</code><a class="headerlink" href="#scrapy.extensions.statsmailer.StatsMailer" title="永久链接至目标">¶</a></dt>
<dd></dd></dl>

<p>这个简单的扩展可用于在域完成抓取时发送通知电子邮件，包括收集到的残缺统计信息。电子邮件将发送给 <a class="reference internal" href="settings.html#std-setting-STATSMAILER_RCPTS"><code class="xref std std-setting docutils literal notranslate"><span class="pre">STATSMAILER_RCPTS</span></code></a> 设置。</p>
<span class="target" id="module-scrapy.extensions.debug"></span></div>
</div>
<div class="section" id="debugging-extensions">
<h3>调试扩展<a class="headerlink" href="#debugging-extensions" title="永久链接至标题">¶</a></h3>
<div class="section" id="stack-trace-dump-extension">
<h4>堆栈跟踪转储扩展<a class="headerlink" href="#stack-trace-dump-extension" title="永久链接至标题">¶</a></h4>
<dl class="py class">
<dt id="scrapy.extensions.debug.StackTraceDump">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.extensions.debug.</code><code class="sig-name descname">StackTraceDump</code><a class="reference internal" href="../_modules/scrapy/extensions/debug.html#StackTraceDump"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.extensions.debug.StackTraceDump" title="永久链接至目标">¶</a></dt>
<dd></dd></dl>

<p>当 <a class="reference external" href="https://en.wikipedia.org/wiki/SIGQUIT">SIGQUIT</a> 或 <a class="reference external" href="https://en.wikipedia.org/wiki/SIGUSR1_and_SIGUSR2">SIGUSR2</a> 接收到信号。转储的信息如下：</p>
<ol class="arabic simple">
<li><p>发动机状态（使用 <code class="docutils literal notranslate"><span class="pre">scrapy.utils.engine.get_engine_status()</span></code> ）</p></li>
<li><p>实时参考（请参见 <a class="reference internal" href="leaks.html#topics-leaks-trackrefs"><span class="std std-ref">使用调试内存泄漏 trackref</span></a> ）</p></li>
<li><p>所有线程的堆栈跟踪</p></li>
</ol>
<p>在转储堆栈跟踪和引擎状态后，  Scrapy   进程继续正常运行。</p>
<p>此扩展仅适用于与POSIX兼容的平台（即，不适用于Windows），因为 <a class="reference external" href="https://en.wikipedia.org/wiki/SIGQUIT">SIGQUIT</a> 和 <a class="reference external" href="https://en.wikipedia.org/wiki/SIGUSR1_and_SIGUSR2">SIGUSR2</a> Windows上没有信号。</p>
<p>至少有两种方法可以将Scrapy <a class="reference external" href="https://en.wikipedia.org/wiki/SIGQUIT">SIGQUIT</a> 信号：</p>
<ol class="arabic">
<li><p>按ctrl-while a scrapy process is running（Linux only？）</p></li>
<li><p>通过运行此命令（假设 <code class="docutils literal notranslate"><span class="pre">&lt;pid&gt;</span></code> 是报废流程的流程ID）：：</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">kill</span> <span class="o">-</span><span class="n">QUIT</span> <span class="o">&lt;</span><span class="n">pid</span><span class="o">&gt;</span>
</pre></div>
</div>
</li>
</ol>
</div>
<div class="section" id="debugger-extension">
<h4>调试器扩展<a class="headerlink" href="#debugger-extension" title="永久链接至标题">¶</a></h4>
<dl class="py class">
<dt id="scrapy.extensions.debug.Debugger">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.extensions.debug.</code><code class="sig-name descname">Debugger</code><a class="reference internal" href="../_modules/scrapy/extensions/debug.html#Debugger"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.extensions.debug.Debugger" title="永久链接至目标">¶</a></dt>
<dd></dd></dl>

<p>调用一个 <a class="reference external" href="https://docs.python.org/3/library/pdb.html" title="(在 Python v3.9)"><span class="xref std std-doc">Python debugger</span></a> 当 <a class="reference external" href="https://en.wikipedia.org/wiki/SIGUSR1_and_SIGUSR2">SIGUSR2</a> 接收到信号。退出调试程序后，碎片处理程序继续正常运行。</p>
<p>有关详细信息，请参阅 <a class="reference external" href="https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/">Debugging in Python</a> .</p>
<p>此扩展仅适用于与POSIX兼容的平台（即不适用于Windows）。</p>
</div>
</div>
</div>
</div>


           </div>
           
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="api.html" class="btn btn-neutral float-right" title="核心API" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="spider-middleware.html" class="btn btn-neutral float-left" title="蜘蛛中间件" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>
        
        &copy; 版权所有 2008–2020, Scrapy developers
      <span class="lastupdated">
        最后更新于 10月 18, 2020.
      </span>

    </p>
  </div>
    
    
    
    Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a
    
    <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a>
    
    provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
  
 
<script type="text/javascript">
!function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","page","once","off","on"];analytics.factory=function(t){return function(){var e=Array.prototype.slice.call(arguments);e.unshift(t);analytics.push(e);return analytics}};for(var t=0;t<analytics.methods.length;t++){var e=analytics.methods[t];analytics[e]=analytics.factory(e)}analytics.load=function(t){var e=document.createElement("script");e.type="text/javascript";e.async=!0;e.src=("https:"===document.location.protocol?"https://":"http://")+"cdn.segment.com/analytics.js/v1/"+t+"/analytics.min.js";var n=document.getElementsByTagName("script")[0];n.parentNode.insertBefore(e,n)};analytics.SNIPPET_VERSION="3.1.0";
analytics.load("8UDQfnf3cyFSTsM4YANnW5sXmgZVILbA");
analytics.page();
}}();

analytics.ready(function () {
    ga('require', 'linker');
    ga('linker:autoLink', ['scrapinghub.com', 'crawlera.com']);
});
</script>


</body>
</html>